# https://pbl.neoscholar.com/list
import requests
from pyquery import PyQuery as pq
import time
import pandas as pd
import os
import json

cookies = {
    "VOLCALB": "14ea67d4d82fb7927e69198e5981c5d9|1716822640|1716822575",
    "VOLCALBCORS": "14ea67d4d82fb7927e69198e5981c5d9|1716822640|1716822575",
}

headers = {
    "Accept": "application/json, text/plain, */*",
    "Accept-Language": "zh-CN,zh;q=0.9,en-US;q=0.8,en;q=0.7",
    "Cache-Control": "no-cache",
    "Connection": "keep-alive",
    # 'Cookie': 'VOLCALB=14ea67d4d82fb7927e69198e5981c5d9|1716822640|1716822575; VOLCALBCORS=14ea67d4d82fb7927e69198e5981c5d9|1716822640|1716822575',
    "Pragma": "no-cache",
    "Referer": "https://pbl.neoscholar.com/list",
    "Sec-Fetch-Dest": "empty",
    "Sec-Fetch-Mode": "cors",
    "Sec-Fetch-Site": "same-origin",
    "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
    "sec-ch-ua": '"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"',
    "sec-ch-ua-mobile": "?0",
    "sec-ch-ua-platform": '"macOS"',
}


result_df = pd.DataFrame()
fileName = "学术"


def fetch_data():
    urls = [
        f"https://pbl.neoscholar.com/api/productpage/keti/searchListV2?standardCode=P0121,P0122,P0124,P0130,P0132&xids=&xid2s=&coursePhases=&coursePhases2=&nationality=&courseCity=&name=&url=https:%2F%2Fpbl.neoscholar.com%2Flist&page={x}&size=10"
        for x in range(1, 10)
    ]
    for url in urls:
        print(url)
        try:
            response = requests.get(url, cookies=cookies, headers=headers).json()
            list1 = response["result"]["ketiList"]["list"]
            list2 = []
            for item in list1:
                list2.append(
                    {
                        "majorsName": item["majorsName"],
                        "desc": item["desc"],
                        "keywords": item["keywords"],
                    }
                )

            global result_df
            result_df = pd.concat([result_df, pd.DataFrame(list2)])
            # time.sleep(1)
        except:
            pass

    save()


def save():
    path = os.path.join(os.path.dirname(__file__), "output", f"{fileName}.xlsx")
    result_df.to_excel(
        path,
        index=False,
        header=["majorsName", "desc", "keywords"],
    )  # 输出总的结果
    print("下载完成")


if __name__ == "__main__":
    fetch_data()
